typedef struct swcache_lock {
  int req, cnt;
} swcache_lock_t;

#ifdef CPE
#ifdef SWCACHE_ENABLE_PROF
#define ASMPROF ""
#else
#define ASMPROF "#"
#endif

#define CMASK(BITS) ((1 << BITS) -1 )
#define ALN_SIZE(SIZE, BITS) ((SIZE) + CMASK(BITS) & ~CMASK(BITS))
#define DR_DAT_SIZE(TBITS, LBITS, HBITS)                        \
  (ALN_SIZE(32 + (8<<HBITS), TBITS) + (1 << TBITS+LBITS+HBITS))
//#define DU_DAT_SIZE(TBITS, LBITS, HBITS)                              \
//  (ALN_SIZE(64 + (8<<HBITS), TBITS) + ((1<<TBITS+LBITS)+1 << HBITS))

#define AMASK(BITS) "((1 << "#BITS") - 1)"
#define AALIGN(SIZE, BITS) "(("SIZE "+" AMASK(BITS) ")&~" AMASK(BITS) ")"
#define DR_CDATA_OFF(HBITS, TBITS) AALIGN("((8<<"#HBITS") + 32)", TBITS)

#define SWCACHE_LOCK(lock) {                    \
    int t0, t1;                                 \
    asm volatile("faaw %0, 0(%2)\n\t"           \
                 "1000:"                        \
                 "ldw %1, 4(%2)\n\t"            \
                 "subl %1, %0, %1\n\t"          \
                 "bne %1, 1000b\n\t"            \
                 : "=&r"(t0), "=&r"(t1)         \
                 : "r"(lock));                  \
  }
#define SWCACHE_UNLOCK(lock) asm volatile("faaw $31, 4(%0)\n\t" :: "r"(lock))

#define dma_and_syn(desc, mem, ldm) {                   \
    int t0;                                             \
    asm("vextw %1, 2, %0\n\t"                           \
        "zapnot %0, 3, %0\n\t"                          \
        "stw $31, 0(%0)\n\t"                            \
        "dma %1, %2, %3\n\t"                            \
        "1:\n\t"                                        \
        "vextw %1, 2, %0\n\t"                           \
        "zapnot %0, 3, %0\n\t"                          \
        "ldw %0, 0(%0)\n\t"                             \
        "beq %0, 1b\n\t"                                \
        : "=&r"(t0) : "r"(desc), "r"(mem), "r"(ldm));   \
  }
                                                                 
#define SWCACHE_INIT(...) SWCACHE_INIT_ASM(__VA_ARGS__)
#define SWCACHE_TADDR(ret, ...) SWCACHE_TADDR_ASM(ret, __VA_ARGS__)
#define SWCACHE_INIT_ASM(PTR, TYPE, TBITS, LBITS, HBITS)                \
  char PTR ## _dat[DR_DAT_SIZE(TBITS, LBITS, HBITS)];                   \
  int *PTR ## _access = PTR ## _dat + 24;                               \
  int *PTR ## _miss = PTR ## _dat + 28;                                 \
  for (int _i_ = 4; _i_ < (1 << HBITS) + 4; _i_ ++)                     \
    ((long*)PTR ## _dat)[_i_] = -1;                                     \
  dma_desc *PTR ## _dma_desc = PTR ## _dat;                             \
  *PTR ## _dma_desc = 0;                                                \
  dma_set_size(PTR ## _dma_desc, sizeof(TYPE) << LBITS);                \
  dma_set_mode(PTR ## _dma_desc, PE_MODE);                              \
  dma_set_op(PTR ## _dma_desc, DMA_GET);                                \
  dma_set_reply(PTR ## _dma_desc, PTR ## _dat + 16);                    \

#define SWCACHE_INIT_U(...) SWCACHE_INIT_U_ASM(__VA_ARGS__)
#define SWCACHE_UPDATE(...) SWCACHE_UPDATE_ASM(__VA_ARGS__)
#define SWCACHE_FLUSH(...) SWCACHE_FLUSH_ASM(__VA_ARGS__)

#define SWCACHE_INIT_U_C(PTR, TYPE, TBITS, LBITS, HBITS, INIT_OP, MERGE_OP, SYN_PTR) \
  struct {                                                              \
    dma_desc get, put;                                                  \
    long tag[1 << HBITS];                                               \
    long miss, access;                                                  \
    TYPE tmp_data[1 << LBITS];                                          \
    TYPE data[1 << HBITS][1 << LBITS];                                  \
    swcache_lock_t *locks;                                              \
  } PTR ## _cache;                                                      \
  PTR ## _cache.get = 0;                                                \
  dma_set_size(&PTR ## _cache.get, sizeof(TYPE) << LBITS);              \
  dma_set_mode(&PTR ## _cache.get, PE_MODE);                            \
  dma_set_op(&PTR ## _cache.get, DMA_GET);                              \
  dma_set_reply(&PTR ## _cache.get, ((void*)&PTR ## _cache.get) + 16);  \
  PTR ## _cache.put = PTR ## _cache.get;                                \
  dma_set_op(&PTR ## _cache.put, DMA_PUT);                              \
  for (int _i_ = 0; _i_ < (1 << HBITS); _i_ ++)                         \
    PTR ## _cache.tag[_i_] = -1;                                        \
  PTR ## _cache.locks = SYN_PTR;

#define DU_DAT_SIZE(TBITS, LBITS, HBITS) 64 + (8<<HBITS) + (1<<HBITS+LBITS+TBITS) + (1<<LBITS+TBITS)
#define DU_GD "0"
#define DU_PD "32"
#define DU_RPL "16"
#define DU_ACC "24"
#define DU_MIS "28"
#define DU_PPSYN "48"
#define DU_TPSYN "56"
#define DU_TAG "64"
#define DU_TBLK(TBITS, LBITS, HBITS) "64+(8<<"#HBITS")"
#define DU_CBLK(TBITS, LBITS, HBITS) "64+(8<<"#HBITS")+(1<<("#LBITS"+"#TBITS"))"

#define SWCACHE_INIT_U_ASM(PTR, TYPE, TBITS, LBITS, HBITS, OP, SYN_PTR) \
  char PTR ## _dat[DU_DAT_SIZE(TBITS, LBITS, HBITS)];                   \
  {                                                                     \
    char *dat = PTR ## _dat;                                            \
    memset(dat, 0, DU_DAT_SIZE(TBITS, LBITS, HBITS));                   \
    dma_desc *get = dat;                                                \
    dma_desc *put = dat + 32;                                           \
    *get = 0;                                                           \
    dma_set_size(get, sizeof(TYPE) << LBITS);                           \
    dma_set_mode(get, PE_MODE);                                         \
    dma_set_op(get, DMA_GET);                                           \
    dma_set_reply(get, dat + 16);                                       \
    *put = *get;                                                        \
    dma_set_op(put, DMA_PUT);                                           \
    swcache_lock_t **ppsyn = dat + 48;                                  \
    *ppsyn = SYN_PTR;                                                   \
    long *tag = dat + 64;                                               \
    for (int _i_ = 0; _i_ < (1 << HBITS); _i_ ++)                       \
      tag[_i_] = -1;                                                    \
  }

#define _MOP(LD, REG, OFF, BP) LD "\t" REG "," OFF "(" BP ")\n\t"
#define _BOP(OP, RA, RB, RC) OP "\t" RA "," RB "," RC "\n\t"

#define SEQOP_0(LD, ST, OP, DST, SRC, OFF, T1, T2, T3, T4, SZ)  \
  _MOP(LD, T1, OFF, SRC)                                        \
  _MOP(LD, T2, OFF, DST)                                        \
  _BOP(OP, T1, T2, T2)                                          \
  _MOP(ST, T2, OFF, DST)

#define SEQOP_1(LD, ST, OP, DST, SRC, OFF, T1, T2, T3, T4, SZ)  \
  _MOP(LD, T1, OFF, SRC)                                        \
  _MOP(LD, T2, OFF, DST)                                        \
  _MOP(LD, T3, SZ "+" OFF, SRC)                                 \
  _MOP(LD, T4, SZ "+" OFF, DST)                                 \
  _BOP(OP, T1, T2, T2)                                          \
  _BOP(OP, T3, T4, T4)                                          \
  _MOP(ST, T2, OFF, DST)                                        \  
  _MOP(ST, T4, OFF, DST)
  
#define FADDD_0(DST, SRC, OFF, T1, T2, T3, T4)                          \
  SEQOP_0("fldd", "fstd", "faddd", DST, SRC, OFF, T1, T2, T3, T4, "8")
#define FADDD_1(DST, SRC, OFF, T1, T2, T3, T4)                          \
  SEQOP_1("fldd", "fstd", "faddd", DST, SRC, OFF, T1, T2, T3, T4, "8")
#define FADDD_2(DST, SRC, OFF, T1, T2, T3, T4)                          \
  SEQOP_0("vldd", "vstd", "vaddd", DST, SRC, OFF, T1, T2, T3, T4, "32")
#define FADDD_3(DST, SRC, OFF, T1, T2, T3, T4)                          \
  SEQOP_1("vldd", "vstd", "vaddd", DST, SRC, OFF, T1, T2, T3, T4, "32")
#define FADDD_4(DST, SRC, OFF, T1, T2, T3, T4)  \
  FADDD_3(DST, SRC, OFF, T1, T2, T3, T4)        \
  FADDD_3(DST, SRC, "64+"OFF, T1, T2, T3, T4)
#define FADDD_5(DST, SRC, OFF, T1, T2, T3, T4)  \
  FADDD_4(DST, SRC, OFF, T1, T2, T3, T4)        \
  FADDD_4(DST, SRC, "128+"OFF, T1, T2, T3, T4)
#define FADDD_6(DST, SRC, OFF, T1, T2, T3, T4)  \
  FADDD_5(DST, SRC, OFF, T1, T2, T3, T4)        \
  FADDD_5(DST, SRC, "256+"OFF, T1, T2, T3, T4)
#define FADDD_7(DST, SRC, OFF, T1, T2, T3, T4)  \
  FADDD_6(DST, SRC, OFF, T1, T2, T3, T4)        \
  FADDD_6(DST, SRC, "512+"OFF, T1, T2, T3, T4)
#define FADDD_8(DST, SRC, OFF, T1, T2, T3, T4)  \
  FADDD_7(DST, SRC, OFF, T1, T2, T3, T4)        \
  FADDD_7(DST, SRC, "1024+"OFF, T1, T2, T3, T4)
#define FADDD_9(DST, SRC, OFF, T1, T2, T3, T4)  \
  FADDD_8(DST, SRC, OFF, T1, T2, T3, T4)        \
  FADDD_8(DST, SRC, "2048+"OFF, T1, T2, T3, T4)
#define FADDD_10(DST, SRC, OFF, T1, T2, T3, T4)  \
  FADDD_9(DST, SRC, OFF, T1, T2, T3, T4)        \
  FADDD_9(DST, SRC, "4096+"OFF, T1, T2, T3, T4)
#define FADDD_11(DST, SRC, OFF, T1, T2, T3, T4)  \
  FADDD_10(DST, SRC, OFF, T1, T2, T3, T4)        \
  FADDD_10(DST, SRC, "8192+"OFF, T1, T2, T3, T4)
#define FADDD_12(DST, SRC, OFF, T1, T2, T3, T4)  \
  FADDD_11(DST, SRC, OFF, T1, T2, T3, T4)        \
  FADDD_11(DST, SRC, "16384+"OFF, T1, T2, T3, T4)

#define FADDD_OFF 32
#define FADDD_INST "vaddd"
#define SWCACHE_UPDATE_ASM(ptr, TYPE, TBITS, LBITS, HBITS, OP, idx, inc) { \
    int t0, t1, t2, t3;                                                 \
    int256 t4;                                                          \
    asm("# T0 for tag offset, T1 last tag, T2 req tag\n\t"              \
        "veqvw $31, $31, %[T4]\n\t"                                     \
        "sll %[IDX], 64-"#LBITS"-"#HBITS", %[T0]\n\t"                   \
        "srl %[T0], 64-"#HBITS", %[T0]\n\t"                             \
        "s8addl %[T0], %[DAT], %[T0]\n\t"                               \
        "ldl %[T1], "DU_TAG"(%[T0])\n\t"                                \
        "bic %[IDX], (1<<"#LBITS")-1,%[T2]\n\t"                         \
        "subl %[T2], %[T1], %[T3]\n\t"                                  \
        "bne %[T3], 1f\n\t"                                             \
        "bsr $31, 4f\n\t"                                               \
        "1:\n\t"                                                        \
        "# Update tag clean reply\n\t"                                  \
        "stl %[T2], "DU_TAG"(%[T0])\n\t"                                \
        "stw $31, "DU_RPL"(%[DAT])\n\t"                                 \
        "ldi %[T2], 1(%[T1])\n\t"                                       \
        "beq %[T2], 4f\n\t"                                             \
        "# T0, T2, T3 free, try lock\n\t"                               \
        "ldl %[T0], "DU_PPSYN"(%[DAT])\n\t"                             \
        "s8addl %[T1], %[T0], %[T0]\n\t"                                \
        "#halt\n\t"                                                     \
        "faaw %[T3], 0(%[T0])\n\t"                                      \
        "#halt\n\t"                                                     \
        "stl %[T0], "DU_TPSYN"(%[DAT])\n\t"                             \
        "2:\n\t"                                                        \
        "ldw %[T2], 4(%[T0])\n\t"                                       \
        "subw %[T2], %[T3], %[T2]\n\t"                                  \
        "bne %[T2], 2b\n\t"                                             \
        "# T0 for tmpline, T1 for memaddr, fetch old line\n\t"          \
        "vldd %[T3], "DU_GD"(%[DAT])\n\t"                               \
        "vinsf %[T1], %[T4], 2, %[T4]\n\t"                              \
        "ldi %[T2], (1<<"#HBITS"<<"#LBITS")-1($31)\n\t"                 \
        "and %[T1], %[T2], %[T2]\n\t"                                   \
        "sll %[T1], "#TBITS", %[T1]\n\t"                                \
        "addl %[T1], %[PTR], %[T1]\n\t"                                 \
        "vinsf %[T1], %[T4], 0, %[T4]\n\t"                              \
        "ldi %[T0], "DU_TBLK(TBITS, LBITS, HBITS)"(%[DAT])\n\t"         \
        "dma %[T3], %[T1], %[T0]\n\t"                                   \
        "2:\n\t"                                                        \
        "ldw %[T3], "DU_RPL"(%[DAT])\n\t"                               \
        "beq %[T3], 2b\n\t"                                             \
        "# T2 for cached data\n\t"                                      \
        "sll %[T2], "#TBITS", %[T2]\n\t"                                \
        "vinsf %[T2], %[T4], 1, %[T4]\n\t"                              \
        "vinsf %[IDX], %[T4], 3, %[T4]\n\t"                             \
        "addl %[T2], %[DAT], %[T2]\n\t"                                 \
        "ldi %[T2], "DU_CBLK(TBITS, LBITS, HBITS)"(%[T2])\n\t"          \
        "# spill: ptr->DAT_1, inc->DAT_2 idx->DAT_3\n\t"                \
        "# T1->T2_2, T2->T2_1, T0->T0_1 to itself\n\t"                  \
        "vinsf %[PTR], %[DAT], 1, %[DAT]\n\t"                           \
        "vinsf %[INC], %[DAT], 2, %[DAT]\n\t"                           \
        "vinsf %[IDX], %[DAT], 3, %[DAT]\n\t"                           \
        "vinsf %[T1], %[T2], 2, %[T2]\n\t"                              \
        "vinsf %[T2], %[T2], 1, %[T2]\n\t"                              \
        "vinsf %[T0], %[T0], 1, %[T0]\n\t"                              \
        "# add T2 to T0, T1, T3 as tmp reg, idx as counter\n\t"         \
        "ldi %[IDX], 1<<("#LBITS"+"#TBITS")($31)\n\t"                   \
        "#halt\n\t"                                                     \
        "3:\n\t"                                                        \
        "vldd %[T1], 0(%[T0])\n\t"                                      \
        "vldd %[T3], 0(%[T2])\n\t"                                      \
        "vldd %[INC], 32(%[T0])\n\t"                                    \
        "vldd %[PTR], 32(%[T2])\n\t"                                    \
        "vaddd %[T1], %[T3], %[T1]\n\t"                                 \
        "vaddd %[INC], %[PTR], %[INC]\n\t"                              \
        "vstd $31, 0(%[T2])\n\t"                                        \
        "vstd $31, 32(%[T2])\n\t"                                       \
        "vstd %[T1], 0(%[T0])\n\t"                                      \
        "vstd %[INC], 32(%[T0])\n\t"                                    \
        "ldi %[IDX], -64(%[IDX])\n\t"                                   \
        "ldi %[T0], 64(%[T0])\n\t"                                      \
        "ldi %[T2], 64(%[T2])\n\t"                                      \
        "bgt %[IDX], 3b\n\t"                                            \
        "# reload T2, T0\n\t"                                           \
        "vextf %[T2], 2, %[T1]\n\t"                                     \
        "vextf %[T2], 1, %[T2]\n\t"                                     \
        "vextf %[T0], 1, %[T0]\n\t"                                     \
        "vextf %[DAT], 1, %[PTR]\n\t"                                   \
        "vextf %[DAT], 2, %[INC]\n\t"                                   \
        "vextf %[DAT], 3, %[IDX]\n\t"                                   \
        "# put back tmpline\n\t"                                        \
        "vldd %[T3], "DU_PD"(%[DAT])\n\t"                               \
        "stw $31, "DU_RPL"(%[DAT])\n\t"                                 \
        "dma %[T3], %[T1], %[T0]\n\t"                                   \
        "2:\n\t"                                                        \
        "ldw %[T3], "DU_RPL"(%[DAT])\n\t"                               \
        "beq %[T3], 2b\n\t"                                             \
        "ldl %[T3], "DU_TPSYN"(%[DAT])\n\t"                             \
        "#halt\n\t"                                                     \
        "faaw $31, 4(%[T3])\n\t"                                        \
        "#halt\n\t"                                                     \
        "4:\n\t"                                                        \
        "# point T0 to area to be updated\n\t"                          \
        "sll %[IDX], 64-"#LBITS"-"#HBITS", %[T0]\n\t"                   \
        "srl %[T0], 64-"#LBITS"-"#HBITS"-"#TBITS", %[T0]\n\t"           \
        "addl %[T0], %[DAT], %[T0]\n\t"                                 \
        "ldi %[T0], "DU_CBLK(TBITS, LBITS, HBITS)"(%[T0])\n\t"          \
        "ldi %[T1], 1<<"#TBITS"($31)\n\t"                               \
        "vinsf %[INC], %[INC], 1, %[INC]\n\t"                           \
        "5:\n\t"                                                        \
        "fldd %[T2], 0(%[INC])\n\t"                                     \
        "fldd %[T3], 0(%[T0])\n\t"                                      \
        "faddd %[T2], %[T3], %[T2]\n\t"                                 \
        "fstd %[T2], 0(%[T0])\n\t"                                      \
        "ldi %[T0], 8(%[T0])\n\t"                                       \
        "ldi %[INC], 8(%[INC])\n\t"                                     \
        "ldi %[T1], -8(%[T1])\n\t"                                      \
        "bgt %[T1], 5b\n\t"                                             \
        "vextf %[INC], 1, %[INC]"                                       \
        : [T0]"=&r"(t0), [T1]"=&r"(t1), [T2]"=&r"(t2), [T3]"=&r"(t3), [T4]"=&r"(t4) \
        : [IDX]"r"(idx), [PTR]"r"(ptr), [INC]"r"(inc), [DAT]"r"(ptr ## _dat) \
        );                                                              \
  }

#define SWCACHE_FLUSH_ASM(PTR, TYPE, TBITS, LBITS, HBITS, OP)   \
  {                                                             \
    char *dat = PTR ## _dat;                                    \
    dma_desc *get = dat;                                        \
    dma_desc *put = dat + 32;                                   \
    swcache_lock_t **ppsyn = dat + 48;                          \
    swcache_lock_t *psyn = *ppsyn;                              \
    long *tag = dat + 64;                                       \
    TYPE *tblk = dat + 64 + (8 << HBITS);                       \
    TYPE (*cblks)[1 << LBITS] = tblk + (1 << LBITS);            \
    for (int _i_ = 0; _i_ < (1 << HBITS); _i_ ++) {             \
      if (tag[_i_] != -1){                                      \
        long lockoff = tag[_i_] >> LBITS;                       \
        swcache_lock_t *blklock = psyn + lockoff;               \
        SWCACHE_LOCK(blklock);                                  \
        dma_and_syn(*get, PTR + tag[_i_], tblk);                \
        for (int _j_ = 0; _j_ < (1 << LBITS); _j_ ++) {         \
          tblk[_j_] += cblks[_i_][_j_];                         \
          cblks[_i_][_j_] = 0;                                  \
        }                                                       \
        dma_and_syn(*put, PTR + tag[_i_], tblk);                \
        SWCACHE_UNLOCK(blklock);                                \
        tag[_i_] = -1;                                          \
      }                                                         \
    }                                                           \
  }

#define SWCACHE_UPDATE_C(PTR, TYPE, TBITS, LBITS, HBITS, INIT_OP, MERGE_OP, idx, _dat_) { \
    long gl_off1 = idx >> LBITS;                                        \
    long off1 = gl_off1 & ((1 << HBITS) - 1);                           \
    long off2 = idx & ((1 << LBITS) - 1);                               \
    PTR ## _cache.access ++;                                            \
    if (PTR ## _cache.tag[off1] != gl_off1) {                           \
      PTR ## _cache.miss ++;                                            \
      long ofirst = PTR ## _cache.tag[off1];                            \
      long nfirst = idx & ~((1 << LBITS) - 1);                          \
      PTR ## _cache.tag[off1] = nfirst;                                 \
      if (ofirst != -1) {                                               \
        SWCACHE_LOCK(PTR ## _cache.locks + (ofirst >> LBITS));          \
        dma_and_syn(PTR ## _cache.get, PTR + ofirst, PTR ## _cache.tmp_data); \
        for (int _i_ = 0; _i_ < (1 << LBITS); _i_ ++) {                 \
          TYPE *ost = PTR ## _cache.tmp_data + _i_;                     \
          TYPE *nst = PTR ## _cache.data[off1] + _i_;                   \
          MERGE_OP(ost, nst);                                           \
          INIT_OP(nst);                                                 \
        }                                                               \
        dma_and_syn(PTR ## _cache.put, PTR + ofirst, PTR ## _cache.tmp_data); \
        SWCACHE_UNLOCK(PTR ## _cache.locks + (ofirst >> LBITS));        \
      }                                                                 \
    }                                                                   \
    MERGE_OP((PTR ## _cache.data[off1] + off2), _dat_);                 \
  }


#define SWCACHE_FLUSH_C(PTR, TYPE, TBITS, LBITS, HBITS, INIT_OP, MERGE_OP) { \
    for (int _j_ = 0; _j_ < (1 << HBITS); _j_ ++){                      \
      long ofirst = PTR ## _cache.tag[_j_];                             \
      if (ofirst != -1) {                                               \
        SWCACHE_LOCK(PTR ## _cache.locks + (ofirst >> LBITS));          \
        dma_and_syn(PTR ## _cache.get, PTR + ofirst, PTR ## _cache.tmp_data); \
        for (int _i_ = 0; _i_ < (1 << LBITS); _i_ ++) {                 \
          TYPE *ost = PTR ## _cache.tmp_data + _i_;                     \
          TYPE *nst = PTR ## _cache.data[_j_] + _i_;                    \
          MERGE_OP(ost, nst);                                           \
          INIT_OP(nst);                                                 \
        }                                                               \
        dma_and_syn(PTR ## _cache.put, PTR + ofirst, PTR ## _cache.tmp_data); \
        SWCACHE_UNLOCK(PTR ## _cache.locks + (ofirst >> LBITS));        \
      }                                                                 \
    }                                                                   \
  }                                                                     

#define SWCACHE_TADDR_ASM(ret, ptr, TYPE, TBITS, LBITS, HBITS, idx)     \
  {                                                                     \
    int t0, t1, t2, t3;                                                 \
    asm("sll %[IDX], 64 - "#LBITS" - "#HBITS", %[TMP1]\n\t"             \
        "srl %[TMP1], 64 - "#HBITS", %[TMP1]\n\t"                       \
        "s8addl %[TMP1], %[DAT], %[TMP1]\n\t"                           \
        "ldl %[RET], 32(%[TMP1])\n\t"                                   \
        "bic %[IDX], (1 <<"#LBITS") - 1, %[TMP0]\n\t"                   \
        "subl %[RET], %[TMP0], %[RET]\n\t"                              \
        "bne %[RET], 3f\n\t"                                            \
        "bsr $31, 1f\n\t"                                               \
        "3:\n\t"                                                        \
        "stl %[TMP0], 32(%[TMP1])\n\t"                                  \
        ASMPROF "ldw %[TMP1], 28(%[DAT])\n\t"                           \
        "#CTAG and TMP1 can be set free here\n\t"                       \
        "stw $31, 16(%[DAT])\n\t"                                       \
        ASMPROF "ldi %[TMP1], 1(%[TMP1])\n\t"                           \
        ASMPROF "stw %[TMP1], 28(%[DAT])\n\t"                           \
        "vldd %[TMP1], 0(%[DAT])\n\t"                                   \
        "sll %[TMP0], "#TBITS", %[TMP0]\n\t"                            \
        "addl %[TMP0], %[PTR], %[TMP0]\n\t"                             \
        "sll %[IDX], 64 - "#LBITS" - "#HBITS", %[RET]\n\t"              \
        "srl %[RET], 64 - "#HBITS", %[RET]\n\t"                         \
        "sll %[RET], "#TBITS "+" #LBITS ", %[RET]\n\t"                  \
        "addl %[RET], %[DAT], %[RET]\n\t"                               \
        "ldi %[RET], "DR_CDATA_OFF(HBITS, TBITS)"(%[RET])\n\t"          \
        "dma %[TMP1], %[TMP0], %[RET]\n\t"                              \
        "#TMP0 is free now\n\t"                                         \
        "2:\n\t"                                                        \
        "ldw %[TMP0], 16(%[DAT])\n\t"                                   \
        "beq %[TMP0], 2b\n\t"                                           \
        "1:\n\t"                                                        \
        ASMPROF "ldw %[TMP1], 24(%[DAT])\n\t"                           \
        "sll %[IDX], 64 - "#LBITS" - "#HBITS", %[TMP0]\n\t"             \
        "srl %[TMP0], 64 - "#LBITS" - "#HBITS" - "#TBITS", %[TMP0]\n\t" \
        "addl %[DAT], %[TMP0], %[RET]\n\t"                              \
        ASMPROF "ldi %[TMP1], 1(%[TMP1])\n\t"                           \
        ASMPROF "stw %[TMP1], 24(%[DAT])\n\t"                           \
        "ldi %[RET], "DR_CDATA_OFF(HBITS, TBITS)"(%[RET])\n\t"          \
        : [RET]"=r"(ret), [TMP0]"=r"(t1), [TMP1]"=&r"(t2)               \
        : [PTR]"r"(ptr), [IDX]"r"(idx), [DAT]"r"(ptr ## _dat));         \
  }
#endif
#ifdef MPE
static inline void *swcache_u_prepare_locks(long length, int lbits) {
  long lmask = (1 << lbits) - 1;
  long nblk = (length + lmask) & ~lmask;
  int *ret = malloc(sizeof(int) * nblk * 2);
  for (int i = 0; i < nblk * 2; i ++) {
    ret[i] = 0;
  }
  return ret;
}
#endif
